# -*- coding: utf-8 -*-
import scrapy

import json
import logging
import datetime
import urllib.parse
from copy import copy, deepcopy
from FDC_spider.constants import PJ_NUM
from FDC_spider.utils.transfer_time import datetime_to_timestamp
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class YichangSpider(scrapy.Spider):
    name = 'yichang'
    allowed_domains = ['219.139.130.120']
    start_urls = ['http://219.139.130.120:7000/pages/commerhouse/sslp.html']
    district_li_url_temp = 'http://219.139.130.120:7000/api/codes/loadJaoArea?_={}'  # 行政区/县列表url
    project_li_url_temp = 'http://219.139.130.120:7000/api/buildInfos/pageHousePreSales?page={}&limit=15&state=2&code=1&ownerArea={}&projectName=&_={}'  # 项目列表url
    # project_li_url_temp = 'http://219.139.130.120:7000/api/buildInfos/pageHousePreSales?page={}&limit=15&state=2&code=1&projectName=&_={}'  # 项目列表url
    project_url_temp = 'http://219.139.130.120:7000/api/buildInfos/getHomePageBuildingInfo?projectId={}'  # 项目详情url
    room_li_url_temp = 'http://219.139.130.120:7000/api/buildInfos/getHouseInfosByPannelNumber?pannelNumber={}&_={}'  # 房号列表url

    """Version 1"""
    # def start_requests(self):
    #     headers = dict(
    #         Referer=self.start_urls[0],
    #     )
    #     yield scrapy.Request(
    #         self.district_li_url_temp.format(self.get_timestamp()),
    #         headers=headers,
    #     )

    """Version 1"""
    # def parse(self, response):
    #     """
    #     获取行政区/县
    #     :param response:
    #     :return:
    #     """
    #     try:
    #         resp = response.body.decode()
    #         resp_dict = json.loads(resp)
    #         district_li = resp_dict['data']
    #         assert resp_dict['code'] == 200 and district_li
    #     except Exception as e:
    #         logger.error("行政区/县获取失败,error_msg:{}".format(e))
    #     else:
    #         # 遍历行政区/县
    #         for district_dict in district_li:
    #             district_name = district_dict.get('dmName')
    #             headers = dict(
    #                 Referer=self.start_urls[0],
    #             )
    #             # 构造首页项目列表请求
    #             project_li_url = self.project_li_url_temp.format(1, urllib.parse.quote(district_name),
    #                                                              self.get_timestamp())
    #             yield scrapy.Request(
    #                 project_li_url,
    #                 headers=headers,
    #                 callback=self.parse_project_li,
    #                 meta=dict(district_name=deepcopy(district_name), page_num=1),
    #             )

    """Version 1"""
    # def parse_project_li(self, response):
    #     """
    #     获取项目列表
    #     :param response:
    #     :return:
    #     """
    #     district_name = copy(response.meta['district_name'])
    #     page_num = copy(response.meta['page_num'])
    #     try:
    #         resp_dict = json.loads(response.body.decode())
    #         code_num = resp_dict['code']
    #         project_li = resp_dict['data'].get('data', list())
    #         total_num = resp_dict['data'].get('total', 0)
    #         assert int(code_num) == 200
    #     except Exception as e:
    #         logger.error('{}-第{}页 项目列表获取出错,error_msg:{}'.format(district_name, page_num, e))
    #     else:
    #         headers = dict(
    #             Referer=self.start_urls[0],
    #         )
    #         if project_li:
    #             # 遍历项目
    #             for project_dict in project_li:
    #                 item_eg = FdcEstateGuidItem()
    #                 item_eg['districtName'] = district_name
    #                 item_eg['projectName'] = project_dict.get('projectName', None)
    #                 item_eg['developerName'] = project_dict.get('companyName', None)
    #                 item_eg['preSalePermit'] = project_dict.get('yszh', None)
    #                 item_eg['preSaleRoomNo'] = project_dict.get('roomNum', None)
    #                 item_eg['preSaleArea'] = project_dict.get('roomArea', None)
    #                 try:
    #                     item_eg['projectId'] = project_dict['xmbh']
    #                 except:
    #                     logger.error('{}-第{}页-{} 项目id获取出错'.format(district_name, page_num, item_eg['projectName'], ))
    #                 else:
    #                     item_bd = FdcBuildingItem()
    #                     item_bd['districtName'] = item_eg['districtName']
    #                     item_bd['projectName'] = item_eg['projectName']
    #                     item_bd['projectId'] = item_eg['projectId']
    #                     item_bd['buildingId'] = project_dict.get('blockNumber', None)
    #                     yield scrapy.Request(
    #                         self.project_url_temp.format(item_eg['projectId'], ),  # 项目详情url不添加时间戳，方便去重处理
    #                         headers=headers,
    #                         callback=self.parse_project,
    #                         meta=dict(item_eg=deepcopy(item_eg), item_bd=deepcopy(item_bd), ),
    #                     )
    #         else:
    #             logger.error('{}-第{}页 项目列表获取为空'.format(district_name, page_num))
    #
    #         # 翻页
    #         total_page_num = int(total_num) // 15 + 1 if int(total_num) % 15 else int(total_num) // 15
    #         if page_num < total_page_num:
    #             page_num += 1
    #             # 构造翻页请求
    #             yield scrapy.Request(
    #                 self.project_li_url_temp.format(page_num, urllib.parse.quote(district_name), self.get_timestamp()),
    #                 headers=headers,
    #                 callback=self.parse_project_li,
    #                 meta=dict(district_name=deepcopy(district_name), page_num=deepcopy(page_num), )
    #             )

    """Version 2"""
    def start_requests(self):
        headers = dict(
            Referer=self.start_urls[0],
        )
        for project_id in range(1, PJ_NUM + 1):
            yield scrapy.Request(
                self.project_url_temp.format(project_id),
                headers=headers,
                dont_filter=False,
                meta=dict(project_id=deepcopy(project_id), )
            )

    """Version 2"""
    def parse(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        project_id = copy(response.meta['project_id'])
        try:
            resp_dict = json.loads(response.body.decode())
            project_dict = resp_dict['data']
            keys_len = len(project_dict.keys())
            assert resp_dict['code'] == 200 and keys_len > 6
        except Exception as e:
            logger.warning('当项目Id为 {} 时,项目不存在或获取出错'.format(project_id))
        else:
            # 项目item
            item_eg = FdcEstateGuidItem()
            item_eg['projectId'] = project_id
            item_eg['projectName'] = project_dict.get('projectName', None)
            item_eg['developerName'] = project_dict.get('companyName', None)
            item_eg['preSalePermit'] = project_dict.get('yszh', None)
            item_eg['preSaleRoomNo'] = project_dict.get('totalrooms', None)
            item_eg['preSaleArea'] = project_dict.get('barea', None)
            item_eg['districtName'] = project_dict.get('houseingArea', None)
            item_eg['projectAddress'] = project_dict.get('houseaddress', None)
            item_eg['planningPermit'] = project_dict.get('plytnum', None)
            item_eg['constructionPermit'] = project_dict.get('cnsnum', None)
            item_eg['landUseRightNum'] = project_dict.get('eusenum', None)
            item_eg['landQuality'] = project_dict.get('landquality', None)
            item_eg['landUse'] = project_dict.get('buse', None)
            item_eg['landUsageTerm'] = project_dict.get('landlimitfromText', None)
            item_eg['greeningRate'] = project_dict.get('greenRate', None)
            item_eg['floorAreaRatio'] = project_dict.get('plotRatio', None)
            item_eg['orientation'] = project_dict.get('orientation', None)
            item_eg['decorateSituation'] = project_dict.get('decoration', None)
            item_eg['propertyArea'] = project_dict.get('propertybarea', None)
            item_eg['propertyFee'] = project_dict.get('propertyfee', None)
            item_eg['parkingRental'] = project_dict.get('parkingrendfee', None)
            item_eg['parkingManagementFee'] = project_dict.get('parkingmanagefee', None)
            item_eg['airSupply'] = project_dict.get('airsupply', None)
            item_eg['waterWorks'] = project_dict.get('waterworks', None)
            item_eg['lift'] = project_dict.get('lift', None)
            item_eg['buildingDensity'] = project_dict.get('buildingdensity', None) if not project_dict.get(
                'buildingdensity', None) else None
            yield item_eg

            # 楼栋item
            item_bd = FdcBuildingItem()
            item_bd['districtName'] = item_eg['districtName']
            item_bd['projectName'] = item_eg['projectName']
            item_bd['projectId'] = item_eg['projectId']
            item_bd['buildingId'] = project_dict.get('blockNumber', None)

            # 获取房号列表信息
            headers = dict(
                Referer='http://219.139.130.120:7000/pages/commerhouse/lpxx.html',
            )
            yield scrapy.Request(
                self.room_li_url_temp.format(item_eg['projectId'], self.get_timestamp()),
                headers=headers,
                callback=self.parse_room_li,
                meta=dict(item_bd=deepcopy(item_bd)),
                priority=10,
            )

    def parse_project(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_bd = copy(response.meta['item_bd'])
        try:
            resp_dict = json.loads(response.body.decode())
            project_dict = resp_dict['data']
            code_num = resp_dict['code']
            assert int(code_num) == 200 and project_dict
        except Exception as e:
            logger.error("{} {}-{} 项目详情获取出错,error_msg:{}".format(response.request.url, item_eg['districtName'],
                                                                 item_eg['projectName'], e))
        else:
            item_eg['districtName'] = project_dict.get('houseingArea', None) if project_dict.get('houseingArea',
                                                                                                 None) else item_eg[
                'districtName']
            item_bd['districtName'] = item_eg['districtName']
            item_eg['projectAddress'] = project_dict.get('houseaddress', None)
            item_eg['planningPermit'] = project_dict.get('plytnum', None)
            item_eg['constructionPermit'] = project_dict.get('cnsnum', None)
            item_eg['landUseRightNum'] = project_dict.get('eusenum', None)
            item_eg['landQuality'] = project_dict.get('landquality', None)
            item_eg['landUse'] = project_dict.get('buse', None)
            item_eg['landUsageTerm'] = project_dict.get('landlimitfromText', None)
            item_eg['greeningRate'] = project_dict.get('greenRate', None)
            item_eg['floorAreaRatio'] = project_dict.get('plotRatio', None)
            item_eg['orientation'] = project_dict.get('orientation', None)
            item_eg['decorateSituation'] = project_dict.get('decoration', None)
            item_eg['propertyArea'] = project_dict.get('propertybarea', None)
            item_eg['propertyFee'] = project_dict.get('propertyfee', None)
            item_eg['parkingRental'] = project_dict.get('parkingrendfee', None)
            item_eg['parkingManagementFee'] = project_dict.get('parkingmanagefee', None)
            item_eg['airSupply'] = project_dict.get('airsupply', None)
            item_eg['waterWorks'] = project_dict.get('waterworks', None)
            item_eg['lift'] = project_dict.get('lift', None)
            item_eg['buildingDensity'] = project_dict.get('buildingdensity', None) if not project_dict.get(
                'buildingdensity', None) else None
            yield item_eg

            # 获取房号列表信息
            headers = dict(
                Referer='http://219.139.130.120:7000/pages/commerhouse/lpxx.html',
            )
            yield scrapy.Request(
                self.room_li_url_temp.format(item_bd['buildingId'], self.get_timestamp()),
                headers=headers,
                callback=self.parse_room_li,
                meta=dict(item_bd=deepcopy(item_bd)),
                priority=10,
            )

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            resp_dict = json.loads(response.body.decode())
            data_dict = resp_dict['data']
            code_num = resp_dict['code']
            assert int(code_num) == 200 and data_dict
        except Exception as e:
            logger.error("{} {}-{} 房号列表信息获取出错,error_msg:{}".format(response.requst.url, item_bd['districtName'],
                                                                   item_bd['projectName'], e))
        else:
            # 解析楼栋信息
            bd_dict = data_dict.get('summaryDto', dict())
            logger.warning(
                '{}-{} 楼栋信息获取为空'.format(item_bd['districtName'], item_bd['projectName'], )) if not bd_dict else True
            item_bd['roomTotalNo'] = bd_dict.get('total', None)
            item_bd['buildingTotalArea'] = bd_dict.get('totalArea', None)
            item_bd['soldRoomNo'] = bd_dict.get('saleTotal', None)
            item_bd['buildingSoldArea'] = bd_dict.get('saleTotalArea', None)
            item_bd['saleableRoomNo'] = bd_dict.get('preSsaleTotal', None)
            item_bd['buildingSaleableArea'] = bd_dict.get('preSaleTotalArea', None)

            # 解析房号列表信息
            rm_li = data_dict.get('roomMain', list())
            if rm_li:
                floor_li = list()
                # 遍历楼层
                for floor_rm_dict in rm_li:
                    floor_li.append(floor_rm_dict['floor'])
                    try:
                        rm_li = floor_rm_dict.get('data', list())
                        assert rm_li
                    except:
                        logger.warning('{}-{}-第{}层 房号列表获取为空或出错'.format(item_bd['districtName'], item_bd['projectName'],
                                                                       floor_rm_dict['floor']))
                    else:
                        check_rm_dict = dict()
                        # 遍历同一楼层下的房号
                        for rm_dict in rm_li:
                            # 判断房号是否为undefined
                            try:
                                houseNumber = rm_dict['houseNumber']
                            except:
                                continue
                            else:
                                item_rm = FdcRoomItem()
                                item_rm['projectId'] = item_bd['projectId']
                                item_rm['buildingId'] = item_bd['buildingId']
                                item_rm['roomNo'] = houseNumber
                                item_rm['nominalFloor'] = rm_dict.get('nominalLevel', None)
                                item_rm['physicsFloor'] = rm_dict.get('cellsNumber', None)
                                item_rm['saleStatus'] = rm_dict.get('saleStatus', None)
                                item_rm['roomArea'] = rm_dict.get('buildArea', None)
                                item_rm['innerArea'] = rm_dict.get('jacketArea', None)
                                item_rm['apportionmentArea'] = rm_dict.get('apportionedArea', None)
                                # 添加到字典，进行去重，同一房号只保留最后(最新)的销售状态
                                check_rm_dict[houseNumber] = item_rm
                        for item in check_rm_dict.values():
                            yield item

                item_bd['floorTotalNo'] = self.get_floor(len(rm_li), floor_li)
                del floor_li
            else:
                item_bd['floorTotalNo'] = None
                logger.warning('{}-{} 房号列表信息获取为空'.format(item_bd['districtName'], item_bd['projectName'], ))
            yield item_bd

    @staticmethod
    def get_timestamp():
        """
        获取本地时间并返回时间戳
        :return: 13位时间戳
        """
        local_date = datetime.datetime.now()
        local_timestamp = datetime_to_timestamp(local_date)
        return local_timestamp

    @staticmethod
    def get_floor(_base, _floor_li):
        """
        判断楼层数
        :param _base: 参考总楼层数 <int>
        :param _floor_li: 楼层列表 <list>
        :return: 总楼层数 <int>
        """
        try:
            floor_li = sorted([int(floor) for floor in _floor_li])
        except:
            return _base
        else:
            return floor_li[-1] if floor_li[-1] > 0 else _base

    # def start_requests(self):
    #     headers = dict(
    #         Referer=self.start_urls[0],
    #     )
    #     page_num = 1
    #     yield scrapy.Request(
    #         self.project_li_url_temp.format(page_num, self.get_timestamp()),
    #         headers=headers,
    #         meta=dict(page_num=deepcopy(page_num)),
    #     )

    # def parse(self, response):
    #     """
    #     获取项目列表
    #     :param response:
    #     :return:
    #     """
    #     page_num = copy(response.meta['page_num'])
    #     try:
    #         resp_dict = json.loads(response.body.decode())
    #         code_num = resp_dict['code']
    #         project_li = resp_dict['data'].get('data', list())
    #         assert int(code_num) == 200
    #     except Exception as e:
    #         logger.error('第{}页 项目列表解析出错,error_msg:{}'.format(page_num, e))
    #     else:
    #         if project_li:
    #             for project_dict in project_li[0:1]:
    #                 item_eg = FdcEstateGuidItem()
    #                 item_eg['districtName'] = project_dict.get('inarea', None)
    #                 item_eg['projectName'] = project_dict.get('projectName', None)
    #                 item_eg['developerName'] = project_dict.get('companyName', None)
    #                 item_eg['preSalePermit'] = project_dict.get('yszh', None)
    #                 item_eg['preSaleRoomNo'] = project_dict.get('roomNum', None)
    #                 item_eg['preSaleArea'] = project_dict.get('roomArea', None)
    #                 try:
    #                     item_eg['projectId'] = project_dict['xmbh']
    #                 except:
    #                     logger.error('第{}页-{} 项目id获取出错'.format(page_num, item_eg['projectName']))
    #                 else:
    #                     item_bd = FdcBuildingItem()
    #                     item_bd['districtName'] = item_eg['districtName']
    #                     item_bd['projectName'] = item_eg['projectName']
    #                     item_bd['projectId'] = item_eg['projectId']
    #                     item_bd['buildingId'] = project_dict.get('blockNumber', None)
    #                     headers = dict(
    #                         Referer=self.start_urls[0],
    #                     )
    #                     yield scrapy.Request(
    #                         self.project_url_temp.format(item_eg['projectId']),
    #                         headers=headers,
    #                         callback=self.parse_project,
    #                         meta=dict(item_eg=deepcopy(item_eg), item_bd=deepcopy(item_bd), ),
    #                     )
    #         else:
    #             logger.error('第{}页 项目列表获取为空'.format(page_num))
    #
    #         # todo 翻页
